package com.mine.crawler.config;

import java.io.BufferedReader;
import java.io.FileReader;
import java.io.IOException;
import java.text.DateFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import javax.xml.parsers.SAXParser;
import javax.xml.parsers.SAXParserFactory;
import javax.xml.soap.Node;

import org.apache.log4j.Logger;
import org.xml.sax.Attributes;
import org.xml.sax.ContentHandler;
import org.xml.sax.ErrorHandler;
import org.xml.sax.InputSource;
import org.xml.sax.Locator;
import org.xml.sax.SAXException;
import org.xml.sax.SAXParseException;
import org.xml.sax.XMLReader;

import com.mine.crawler.beans.MySqlDBCategoryNodeLogical;
import com.mine.crawler.beans.MySqlDBCategoryParentTreeLogical;
import com.mine.crawler.beans.MySqlDBCategoryTable;
import com.mine.crawler.beans.MySqlDBRelatedCategoryNode;

public class DMOZ_XMLProcessor
{
	private Logger log = Logger.getLogger(DMOZ_XMLProcessor.class);
	private static Logger logXml = Logger.getLogger("xml."+DMOZ_XMLProcessor.class);
	private static String structure_filename = "structure.rdf.u8.h2000";
	private static String content_filename = "content.rdf.u8.h2000";
	
	private static String struct_abs_filename = InitializeCrawlerApplication.InitConfig.getProperty("DMOZ_Directory_Path") 
		+System.getProperty("file.separator") + "structure.rdf.u8.h100K";
	
	private static String content_abs_filename = InitializeCrawlerApplication.InitConfig.getProperty("DMOZ_Directory_Path") 
	+System.getProperty("file.separator") + "content.rdf.u8.h2000";

	public static void XMLProcessing()
	{
		//if(Constants.DEBUG_CONFIG_DMOZ_XML_PROCESSOR)
			logXml.info("Struct file Path = "+struct_abs_filename);
			logXml.info("Content file Path = "+content_abs_filename);
			
		SAXParserFactory spf = SAXParserFactory.newInstance();

		// Create the XMLReader to be used to parse the document.
		XMLReader StructFilereader = null, ContentFilereader   = null;
		try
		{
			SAXParser struct_parser = spf.newSAXParser();
			SAXParser content_parser = spf.newSAXParser();
			StructFilereader = struct_parser.getXMLReader();
			ContentFilereader = content_parser.getXMLReader();
		} 
		catch (Exception e)
		{
			logXml.error(e);
			System.exit(1);
		}

		// Specify the error handler and the content handler.
		StructFilereader.setErrorHandler(new MyErrorHandler());
		StructFilereader.setContentHandler(new XMLStructureCTHandler());
		
		ContentFilereader.setErrorHandler(new MyErrorHandler());
		ContentFilereader.setContentHandler(new XMLContentCTHandler());
		// Use the XMLReader to parse the entire file.
		try
		{
			FileReader struct_fr = new FileReader(struct_abs_filename);
            BufferedReader struct_br = new BufferedReader(struct_fr);
			InputSource struct_is = new InputSource(struct_br);
			//Struct File Reading starts here 
			StructFilereader.parse(struct_is);
			
			FileReader content_fr = new FileReader(content_abs_filename);
            BufferedReader content_br = new BufferedReader(content_fr);
			InputSource content_is = new InputSource(content_br);
			//Content File Reading starts here
			ContentFilereader.parse(content_is);
		} 
		catch (SAXException e)
		{
			logXml.error(e);
			System.exit(1);
		} 
		catch (IOException e)
		{
			logXml.error(e);
			System.exit(1);
		}
	}
}

enum XML_TAGs
{
	Node_Topic("Topic"), Att_rid("r:id"), Node_catId("catid"), Node_Title("d:Title"), 
	Node_lastUpdate("lastUpdate"), Node_Description("d:Description"), Node_narrow("narrow"),
	Node_narrow2("narrow2"),
	Att_resource("r:resource"), Node_altLang("altlang"), Node_related("related"),
	Node_Alias("Alias"), Node_ExternalPage("ExternalPage"), Att_about("about"),
	Node_Link_topic("topic");
	
	private String value;
	XML_TAGs(String local_val)
	{
		value = local_val;
	}
	
	String getValue()
	{
		return value;
	}
}

class XMLContentCTHandler implements ContentHandler
{
	private Locator locator;
	private Logger logXml = Logger.getLogger("xml."+XMLContentCTHandler.class);
	private String strNodeValue;
	
	//private int linkTable_local_link_id;
	
	private int linkTable_local_cat_id;
	private String linkTable_local_link_url;
	private String linkTable_local_link_title;
	private String linkTable_local_link_description;
	private String linkTable_local_cat_name;
	
	@Override
	public void characters(char[] ch, int start, int length) throws SAXException
	{
		if(Constants.DEBUG_CONFIG_DMOZ_XML_PROCESSOR_CONTENT)
		{	
			logXml.info(locator.getLineNumber()
				+ "_Character data");
			//showCharacters(ch, start, length);
		}
		
		strNodeValue = new String(ch,start,length);
	}

	@Override
	public void endDocument() throws SAXException
	{
		// TODO Auto-generated method stub
		
	}

	@Override
	public void endElement(String namespaceURI, String localName, String qName)
			throws SAXException
	{
		// TODO Auto-generated method stub
		if(Constants.DEBUG_CONFIG_DMOZ_XML_PROCESSOR_CONTENT)
		{	

		logXml.info(locator.getLineNumber()
				+ "_Closing tag of an element");
		logXml.info("       Namespace: " + namespaceURI);
		logXml.info("      Local name: " + localName);
		logXml.info("  Qualified name: " + qName);
		}
		
		else if (qName.matches(XML_TAGs.Node_catId.getValue()))
		{
			linkTable_local_cat_id = Integer.parseInt(strNodeValue);
		}

		else if (qName.matches(XML_TAGs.Node_Description.getValue()) )
		{
			linkTable_local_link_description = strNodeValue;
		}
		
		else if (qName.matches(XML_TAGs.Node_Link_topic.getValue()))
		{
			linkTable_local_cat_name = strNodeValue;
		}
		
		else if(qName.matches(XML_TAGs.Node_Title.getValue()))
		{
			linkTable_local_link_title = strNodeValue;
		}
		
		else if(qName.matches(XML_TAGs.Node_ExternalPage.getValue()))
		{
			boolean DEBUG = false;
			
			if(DEBUG)
			logXml.info("Printing link Object - "  
					+"cat_id="+linkTable_local_cat_id
					+" link="+linkTable_local_link_url
					+" title="+linkTable_local_link_title
					+" desc="+linkTable_local_link_description
					+" topic="+linkTable_local_cat_name);
			String query = "insert into link (category_id, link, " +
					"link_title, link_description) values("+
					linkTable_local_cat_id+", "+linkTable_local_link_url+", "+
					linkTable_local_link_title+", "+linkTable_local_link_description+")";
			logXml.info(query);
		}
	}

	@Override
	public void endPrefixMapping(String prefix) throws SAXException
	{
		// TODO Auto-generated method stub
		
	}

	@Override
	public void ignorableWhitespace(char[] ch, int start, int length)
			throws SAXException
	{
		// TODO Auto-generated method stub
		
	}

	@Override
	public void processingInstruction(String target, String data)
			throws SAXException
	{
		// TODO Auto-generated method stub
		
	}

	@Override
	public void setDocumentLocator(Locator locator)
	{
		this.locator = locator;
		if(Constants.DEBUG_CONFIG_DMOZ_XML_PROCESSOR_CONTENT)
		{	
			logXml.info(locator.getLineNumber()
				+ "_Document ID: " + locator.getSystemId());
		}
	}

	@Override
	public void skippedEntity(String name) throws SAXException
	{
		// TODO Auto-generated method stub
		
	}

	@Override
	public void startDocument() throws SAXException
	{
		if(Constants.DEBUG_CONFIG_DMOZ_XML_PROCESSOR_CONTENT)
		{	
			logXml.info(locator.getLineNumber()
				+ "_Document parse started");
		}
	}

	@Override
	public void startElement(String namespaceURI, String localName, String qName,
			Attributes atts) throws SAXException
	{
		// TODO Auto-generated method stub
		strNodeValue = "";
		if(Constants.DEBUG_CONFIG_DMOZ_XML_PROCESSOR_CONTENT)
		{	

			logXml.info(locator.getLineNumber()
				+ "_Opening tag of an element");
			logXml.info("       Namespace: " + namespaceURI);
			logXml.info("      Local name: " + localName);
			logXml.info("  Qualified name: " + qName);
			for (int i = 0; i < atts.getLength(); i++)
			{
				logXml.info("       Attribute: " + atts.getQName(i)
						+ "=\"" + atts.getValue(i) + "\"");
			}
		}
		
		/*else if (qName.matches((XML_TAGs.Node_Topic.getValue())) )
		{
		}*/
		else if (qName.matches((XML_TAGs.Node_catId.getValue())))
		{
			linkTable_local_cat_id = 0;
		}
		else if (qName.matches(XML_TAGs.Node_ExternalPage.getValue()))
		{
			for (int i = 0; i < atts.getLength(); i++)
			{
				if(atts.getQName(i).matches((XML_TAGs.Att_about.getValue())) )
					//logXml.info("     Attribute: " + atts.getQName(i) + " = "+atts.getValue(i));
					linkTable_local_link_url = atts.getValue(i);
				else 
					logXml.info("Attribute: " + atts.getQName(i)+ "=" + atts.getValue(i));
			}
		}
		
		else if (qName.matches(XML_TAGs.Node_Description.getValue()))
		{
			linkTable_local_link_description = null;
		}
		
		else if (qName.matches(XML_TAGs.Node_Link_topic.getValue()))
		{
			linkTable_local_cat_name = null;
		}
		
		else if(qName.matches(XML_TAGs.Node_Title.getValue()))
		{
			linkTable_local_link_title = null;
		}
	}

	@Override
	public void startPrefixMapping(String prefix, String uri)
			throws SAXException
	{
		// TODO Auto-generated method stub
		
	}
	
}

class XMLStructureCTHandler implements ContentHandler
{
	private Locator locator;
	private Logger logXml = Logger.getLogger("xml."+XMLStructureCTHandler.class);
	private String strNodeValue;
	
	private int catTable_local_cat_id;
	private String catTable_local_title_node;
	private String catTable_local_descr;
	private Date catTable_local_date;
	private String catTable_local_aeTitle;
	private String catTable_local_resource;
	private String catTable_local_related;
	private DateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");

	// UPDATE FROM HERE - create map/list of related category
	private List<String> catTable_local_related_list = new ArrayList<String>();
	private List<String> catTable_local_altLang_list = new ArrayList<String>();
	private MySqlDBCategoryParentTreeLogical NodeTree = new MySqlDBCategoryParentTreeLogical("");
	
	private Map<Integer, MySqlDBRelatedCategoryNode> category_map = 
		new HashMap<Integer, MySqlDBRelatedCategoryNode> ();
	
	/**
	 * The name and of the SAX document and the current location within the
	 * document.
	 */
	public void setDocumentLocator(Locator locator)
	{
		this.locator = locator;
		if(Constants.DEBUG_CONFIG_DMOZ_XML_PROCESSOR_STRUCT)
		{	
			logXml.info(locator.getLineNumber()
				+ "_Document ID: " + locator.getSystemId());
		}
	}

	/** The parsing of a document has started.. */

	public void startDocument()
	{
		if(Constants.DEBUG_CONFIG_DMOZ_XML_PROCESSOR_STRUCT)
		{	
			logXml.info(locator.getLineNumber()
				+ "_Document parse started");
		}
	}

	/** The parsing of a document has completed.. */
	public void endDocument()
	{
		if(Constants.DEBUG_CONFIG_DMOZ_XML_PROCESSOR_STRUCT)
		{	
			logXml.info(locator.getLineNumber()
				+ "_Document parse ended");
		}
		NodeTree.AddRelatedCategoryUsingCatMap(category_map);
		//NodeTree.printTree();
		// Remove following code after testing
		
	}

	/** The start of a namespace scope */
	public void startPrefixMapping(String prefix, String uri)
	{
		if(Constants.DEBUG_CONFIG_DMOZ_XML_PROCESSOR_STRUCT)
		{	
			logXml.info(locator.getLineNumber()
				+ "_Namespace scope begins");
			logXml.info("     " + prefix + "=\"" + uri + "\"");
		}
	}

	/** The end of a namespace scope */
	public void endPrefixMapping(String prefix)
	{
		if(Constants.DEBUG_CONFIG_DMOZ_XML_PROCESSOR_STRUCT)
		{	
			logXml.info(locator.getLineNumber()
					+ "_Namespace scope ends");
			logXml.info("     " + prefix);
		}
	}

	/** The opening tag of an element. */
	public void startElement(String namespaceURI, String localName,
			String qName, Attributes atts)
	{
		strNodeValue = "";
		if(Constants.DEBUG_CONFIG_DMOZ_XML_PROCESSOR_STRUCT)
		{	

			logXml.info(locator.getLineNumber()
				+ "_Opening tag of an element");
			logXml.info("       Namespace: " + namespaceURI);
			logXml.info("      Local name: " + localName);
			logXml.info("  Qualified name: " + qName);
			for (int i = 0; i < atts.getLength(); i++)
			{
				logXml.info("       Attribute: " + atts.getQName(i)
						+ "=\"" + atts.getValue(i) + "\"");
			}
		}

		else if (qName.matches(XML_TAGs.Node_Title.getValue()))
		{
			
		}
		
		else if (qName.matches((XML_TAGs.Node_Topic.getValue())) )
		{
			for (int i = 0; i < atts.getLength(); i++)
			{
				if(atts.getQName(i).matches((XML_TAGs.Att_rid.getValue())) )
					//logXml.info("     Attribute: " + atts.getQName(i) + " = "+atts.getValue(i));
					catTable_local_title_node = atts.getValue(i);
				else 
					logXml.info("Attribute: " + atts.getQName(i)+ "=" + atts.getValue(i));
			}
		}
		
		else if (qName.matches((XML_TAGs.Node_narrow.getValue())) ||
				qName.matches((XML_TAGs.Node_narrow2.getValue())))
		{
			for (int i = 0; i < atts.getLength(); i++)
			{
				if(atts.getQName(i).matches((XML_TAGs.Att_resource.getValue())) )
					//logXml.info("     Attribute: " + atts.getQName(i) + " = "+atts.getValue(i));
					catTable_local_resource = atts.getValue(i);
				else 
					logXml.info("Attribute: " + atts.getQName(i)+ "=" + atts.getValue(i));
			}
		}
		
		else if (qName.matches((XML_TAGs.Node_related.getValue())) )
		{
			for (int i = 0; i < atts.getLength(); i++)
			{
				if(atts.getQName(i).matches((XML_TAGs.Att_resource.getValue())) )
					//logXml.info("     Attribute: " + atts.getQName(i) + " = "+atts.getValue(i));
					catTable_local_related = atts.getValue(i);
				else 
					logXml.info("Attribute: " + atts.getQName(i)+ "=" + atts.getValue(i));
			}			
		}
		
		else if(qName.matches(XML_TAGs.Node_altLang.getValue()))
		{
			for (int i = 0; i < atts.getLength(); i++)
			{
				if(atts.getQName(i).matches((XML_TAGs.Att_resource.getValue())) )
					//logXml.info("     Attribute: " + atts.getQName(i) + " = "+atts.getValue(i));
					catTable_local_aeTitle = atts.getValue(i);
				else 
					logXml.info("Attribute: " + atts.getQName(i)+ "=" + atts.getValue(i));
			}						
		}
		
		else if (qName.matches(XML_TAGs.Node_catId.getValue()))
		{
			catTable_local_cat_id = 0;
		}
		
		else if (qName.matches(XML_TAGs.Node_lastUpdate.getValue()))
		{
			catTable_local_date = null;
		}
		else if (qName.matches(XML_TAGs.Node_Description.getValue()))
		{
			catTable_local_descr = null;
		}
	}
	/** The closing tag of an element. */
	public void endElement(String namespaceURI, String localName,
			String qName)
	{
		if(Constants.DEBUG_CONFIG_DMOZ_XML_PROCESSOR_STRUCT)
		{	

		logXml.info(locator.getLineNumber()
				+ "_Closing tag of an element");
		logXml.info("       Namespace: " + namespaceURI);
		logXml.info("      Local name: " + localName);
		logXml.info("  Qualified name: " + qName);
		}
		
		else if (qName.matches(XML_TAGs.Node_catId.getValue()))
		{
			catTable_local_cat_id = Integer.parseInt(strNodeValue);
		}
		
		else if (qName.matches((XML_TAGs.Node_related.getValue())) )
		{
			//logXml.info("cat="+catTable_local_title_node
						//+ " related_cat="+ catTable_local_related);
			catTable_local_related_list.add(catTable_local_related);
		}	
		else if (qName.matches(XML_TAGs.Node_Title.getValue()))
		{
			
		}
		
		else if(qName.matches(XML_TAGs.Node_lastUpdate.getValue()))
		{
			try
			{
				catTable_local_date = df.parse( strNodeValue );
				//logXml.info("date - "+strNodeValue);
			}
			catch (ParseException e)
			{
				// TODO Auto-generated catch block
				logXml.error(e);
				//e.printStackTrace();
			}
		}

		else if (qName.matches(XML_TAGs.Node_Description.getValue()))
		{
			catTable_local_descr = strNodeValue;
		}
		
		else if (qName.matches(XML_TAGs.Node_narrow.getValue()) ||
					qName.matches((XML_TAGs.Node_narrow2.getValue())))
		{
			NodeTree.AddChildrenFromList(new MySqlDBCategoryNodeLogical(catTable_local_resource));
		}
		
		else if(qName.matches(XML_TAGs.Node_altLang.getValue()))
		{
			catTable_local_altLang_list.add(catTable_local_aeTitle);
		}
		
		else if(qName.matches((XML_TAGs.Node_Topic.getValue())) )
		{
			//boolean DEBUG = true;
			
			//if(DEBUG)
			MySqlDBCategoryNodeLogical node = new MySqlDBCategoryNodeLogical(catTable_local_title_node);
			MySqlDBCategoryNodeLogical currentNode = NodeTree.search(node); 
			MySqlDBCategoryNodeLogical parentNode = null;
			if(currentNode == null && catTable_local_cat_id>1)
			{
				currentNode = node;
			}
			if(!currentNode.equals(null)|| currentNode.getCategory_id() >1)
			{
				currentNode.setCategory_id(catTable_local_cat_id);
				currentNode.setRelated(catTable_local_related_list);
				currentNode.setAltLangList(catTable_local_altLang_list);
				logXml.info(node.toString());
			}
			if(currentNode.getCategory_id() >1)
				parentNode = NodeTree.searchFileRelatedParent(currentNode); 

			logXml.info("Printing link Object - "  
					+"cat_Title="+catTable_local_title_node
					+" cat_id="+catTable_local_cat_id
					+" last Update="+catTable_local_date
					+" description="+catTable_local_descr
					);
			if(parentNode == null)
			{
				String query1 = "insert into "+com.mine.crawler.beans.Constants.MySql_CATEGORY_TABLE+
			"(category_id, title, description, lastUpdate) values ("+
			catTable_local_cat_id+", "+catTable_local_title_node+", "+
			catTable_local_descr+", "+catTable_local_date+")";
			logXml.info(query1);
			}
			
			else 
			{
			String query2 = "insert into "+com.mine.crawler.beans.Constants.MySql_CATEGORY_TABLE+
				"(category_id, title, description, lastUpdate, parent_category) values ("+
				catTable_local_cat_id+", "+catTable_local_title_node+", "+
				catTable_local_descr+", "+catTable_local_date+ ","+ parentNode.getCategory_id()+")";
				logXml.info(query2);
			}
			
			category_map.put( new Integer(catTable_local_title_node.hashCode()), 
					new MySqlDBRelatedCategoryNode(catTable_local_title_node, catTable_local_cat_id) );
		}
		
	}

	/** Character data. */
	public void characters(char[] ch, int start, int length)
	{
		if(Constants.DEBUG_CONFIG_DMOZ_XML_PROCESSOR_STRUCT)
		{	
			logXml.info(locator.getLineNumber()
				+ "_Character data");
			//showCharacters(ch, start, length);
		}
		strNodeValue = new String(ch,start,length);
	}

	/** Ignorable whitespace character data. */
	public void ignorableWhitespace(char[] ch, int start, int length)
	{
		if(Constants.DEBUG_CONFIG_DMOZ_XML_PROCESSOR_STRUCT)
		{	
			logXml.info(locator.getLineNumber() + "_Whitespace");
			//showCharacters(ch, start, length);
		}
	}

	/** Processing Instruction */
	public void processingInstruction(String target, String data)
	{
		if(Constants.DEBUG_CONFIG_DMOZ_XML_PROCESSOR_STRUCT)
		{	

		logXml.info(locator.getLineNumber()
				+ "_Processing Instruction");
		logXml.info("         Target: " + target);
		logXml.info("           Data: " + data);
		}
	}

	/** A skipped entity. */
	public void skippedEntity(String name)
	{
		if(Constants.DEBUG_CONFIG_DMOZ_XML_PROCESSOR_STRUCT)
		{	

		logXml.info(locator.getLineNumber()
				+ "_Skipped Entity");
		logXml.info("           Name: " + name);
		}
	}

	/**
	 * Internal method to format arrays of characters so the special
	 * whitespace characters will show.
	 */
	public void showCharacters(char[] ch, int start, int length)
	{
		if(Constants.DEBUG_CONFIG_DMOZ_XML_PROCESSOR)
		{	

		logXml.info("        \"");
		for (int i = start; i < start + length; i++)
			switch (ch[i])
			{
			case '\n':
				logXml.info("\\n");
				break;
			case '\r':
				logXml.info("\\r");
				break;
			case '\t':
				logXml.info("\\t");
				break;
			default:
				logXml.info(ch[i]);
				break;
			}
		logXml.info("\"");
	}
	}
}

class MyErrorHandler implements ErrorHandler
{
	private Logger logXml = Logger.getLogger("xml."+MyErrorHandler.class);
	
	public void warning(SAXParseException e) throws SAXException
	{
		logXml.warn("Warning", e);
		throw (e);
	}

	public void error(SAXParseException e) throws SAXException
	{
		logXml.warn("Error", e);
		throw (e);
	}

	public void fatalError(SAXParseException e) throws SAXException
	{
		logXml.warn("Fatal Error", e);
		throw (e);
	}

	private void show(String type, SAXParseException e)
	{
		if(Constants.DEBUG_CONFIG_DMOZ_XML_PROCESSOR)
		{	

		logXml.info(type + ": " + e.getMessage());
		logXml.info("Line " + e.getLineNumber() + " Column "
				+ e.getColumnNumber());
		logXml.info("System ID: " + e.getSystemId());
		}
	}
}
